import scrapy
from douban.items import DoubanItem
from scrapy.http import Request

class DoubanspiderSpider(scrapy.Spider):
    name = 'doubanspider'
    #allowed_domains = ['https://movie.douban.com/top250']
    start_urls = ['https://movie.douban.com/top250']
    ua='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'

    def parse(self, response):
        ranks=response.xpath('//div[@class="pic"]/em/text()').extract()  #排名
        names=response.xpath('//div[@class="hd"]/a/span[1]/text()').extract() #名称
        comments=response.xpath('//p[@class="quote"]/span/text()').extract()   #内容
        links=response.xpath('//div[@class="hd"]/a/@href').extract()         #链接
        for ind,link in enumerate(links):
            rank=ranks[ind]
            name=names[ind]
            comment=comments[ind]
            #通过meta在request和response间传递数据
            yield scrapy.Request(link,callback=self.parse_next,meta={'rank':rank,
                                                                     'name':name,
                                                                     'comment':comment,
                                                                     'dont_redirect': True,
                                                                     'handle_httpstatus_list': [302]})
            #通过后一页来构建下一个主页面
            next_page=response.xpath('//span[@class="next"]/a/@href').extract_first()  #后页链接
            urs='https://movie.douban.com/top250'
            if next_page:
                yield scrapy.Request(url=urs+next_page,callback=self.parse,headers={'User-Agent':self.ua})
    def parse_next(self,responses):
        rank=responses.meta['rank']
        name=responses.meta['name']
        comment=responses.meta['comment']
        director=responses.xpath('//div[@id="info"]//span[@class="attrs"]/a/text()').extract_first() #导演
        time=responses.xpath('//span[@property="v:initialReleaseDate"]/text()').extract_first()  #上映时间
        long=responses.xpath('//span[@property="v:runtime"]/text()').extract_first()  #片长
        score=responses.xpath('//strong[@class="ll rating_num"]/text()').extract_first()  #评分
        introduction=responses.xpath('//span[@property="v:summary"]/text()').extract_first().strip()  #剧情简介
        item=DoubanItem()
        item['rank']=rank
        item['name']=name
        item['comment']=comment
        item['director']=director
        item['time']=time
        item['long']=long
        item['score']=score
        item['introduction']=introduction
        yield item



